[ { "memory": 1414283520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1420402944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1426522368, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1428881664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1431240960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1433600256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1439719680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1445839104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1448198400, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1454317824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1460437248, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1466556672, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1472676096, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1475035392, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1481154816, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1483514112, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1485873408, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1488232704, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1490592000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1496711424, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1502830848, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1505190144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1507549440, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1513668864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1516028160, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1522147584, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1524506880, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1530626304, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1536745728, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1539105024, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1541464320, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1543823616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1546182912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1552302336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1554661632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1560781056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1563140352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1565499648, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1567858944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1570218240, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1572577536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1574936832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1577296128, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1583415552, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1589534976, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1595654400, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1601773824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1604133120, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1606492416, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1608851712, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 1 } }, { "memory": 1614971136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 1, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1621090560, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 1, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1627209984, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1629569280, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1631928576, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1638048000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1640407296, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1646526720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1648886016, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1655005440, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1661124864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1663484160, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1665843456, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1668202752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1674322176, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1676681472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1682800896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1688920320, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1695039744, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1697399040, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1703518464, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1709637888, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1711997184, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1718116608, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1720475904, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1726595328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1732714752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1738834176, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1741193472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1743552768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1749672192, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1755791616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1758150912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1764270336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1770389760, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1772749056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1775108352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1781227776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1787347200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1793466624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 1, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1799586048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1805705472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1808064768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1814184192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1820303616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1826423040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1832542464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1834901760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1841021184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1843380480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1849499904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1855619328, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1857978624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1864098048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1870217472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1876336896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 1, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1878696192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1884815616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1890935040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1893294336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1895653632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 1, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1901773056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1904132352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1906491648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1912611072, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1918730496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1924849920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1927209216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 1, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1929568512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1935687936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1941807360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1947926784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1950286080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1952645376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1958764800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1961124096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1967243520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1973362944, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1975722240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1981841664, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1984200960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1986560256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1992679680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 1998799104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2001158400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2007277824, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2009637120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2015756544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2021875968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2024235264, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2026594560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2032713984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2035073280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 1, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2041192704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 1, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2043552000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2045911296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2052030720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 1, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2058150144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2064269568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2066628864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2068988160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2075107584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 1, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2077466880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2083586304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2085945600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2092065024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 1, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2098184448, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2104303872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2106663168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 1, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2112782592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 1, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2118902016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2121261312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2127380736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2133500160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2135859456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2141978880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2144338176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2146697472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2149056768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2151416064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2153775360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2156134656, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2158493952, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2160853248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2163212544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2165571840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2167931136, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 1, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2170290432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2172649728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2175009024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 1, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2177368320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2179727616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 1, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2182086912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2184446208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2186805504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2189164800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2191524096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2193883392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 1, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2200002816, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2202362112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 1, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2204721408, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2207080704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2209440000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 1, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2211799296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 1, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2214158592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2220278016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 1, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2222637312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2224996608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2227355904, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2229715200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2232074496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2238193920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 1, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2240553216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2242912512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2245271808, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2247631104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 1, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2249990400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2252349696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2254708992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 1, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2257068288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2259427584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2261786880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 1, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2264146176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2270265600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2272624896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2274984192, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2277343488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2279702784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2282062080, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 1, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2284421376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 1, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2290540800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2292900096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 1, "layer.31.mlp.down_proj": 2 } }, { "memory": 2299019520, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2301378816, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2303738112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2309857536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2315976960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2318336256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2320695552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2323054848, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2329174272, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2335293696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2337652992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2343772416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2349891840, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2356011264, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2362130688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2364489984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2370609408, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2372968704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2375328000, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2377687296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2380046592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2386166016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2392285440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2394644736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2397004032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2403123456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2405482752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2411602176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2413961472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2420080896, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2426200320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2428559616, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2430918912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2433278208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2435637504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2441756928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2444116224, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2450235648, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2452594944, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2454954240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2457313536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2459672832, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2462032128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2464391424, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2466750720, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2472870144, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2478989568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2485108992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2491228416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2493587712, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2495947008, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2498306304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 2 } }, { "memory": 2504425728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 2, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2510545152, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 2, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2516664576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2519023872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2521383168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2527502592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2529861888, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2535981312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2538340608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2544460032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2550579456, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2552938752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2555298048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2557657344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2563776768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2566136064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2572255488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2578374912, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2584494336, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2586853632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2592973056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2599092480, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2601451776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2607571200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2609930496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2616049920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2622169344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2628288768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2630648064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2633007360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2639126784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2645246208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2647605504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2653724928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2659844352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2662203648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2664562944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2670682368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2676801792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2682921216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 2, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2689040640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2695160064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2697519360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2703638784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2709758208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2715877632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2721997056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2724356352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2730475776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2732835072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2738954496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2745073920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2747433216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2753552640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2759672064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2765791488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 2, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2768150784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2774270208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2780389632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2782748928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2785108224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 2, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2791227648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2793586944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2795946240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2802065664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2808185088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2814304512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2816663808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 2, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2819023104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2825142528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2831261952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2837381376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2839740672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2842099968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2848219392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2850578688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2856698112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2862817536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2865176832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2871296256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2873655552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2876014848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2882134272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2888253696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2890612992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2896732416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2899091712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2905211136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2911330560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2913689856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2916049152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2922168576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2924527872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 2, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2930647296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 2, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2933006592, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2935365888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2941485312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 2, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2947604736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2953724160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2956083456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2958442752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2964562176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 2, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2966921472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2973040896, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2975400192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2981519616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 2, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2987639040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2993758464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 2996117760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 2, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3002237184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 2, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3008356608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3010715904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3016835328, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3022954752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3025314048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3031433472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3033792768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3036152064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3038511360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3040870656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3043229952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3045589248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3047948544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3050307840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3052667136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3055026432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3057385728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 2, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3059745024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3062104320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3064463616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 2, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3066822912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3069182208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 2, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3071541504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3073900800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3076260096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3078619392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3080978688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3083337984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 2, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3089457408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3091816704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 2, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3094176000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3096535296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3098894592, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 2, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3101253888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 2, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3103613184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3109732608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 2, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3112091904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3114451200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3116810496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3119169792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3121529088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3127648512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 2, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3130007808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3132367104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3134726400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3137085696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 2, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3139444992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3141804288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3144163584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 2, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3146522880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3148882176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3151241472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 2, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3153600768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3159720192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3162079488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3164438784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3166798080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3169157376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3171516672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 2, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3173875968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 2, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3179995392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3182354688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 2, "layer.31.mlp.down_proj": 3 } }, { "memory": 3188474112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3190833408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3193192704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3199312128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3201671424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3207790848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3210150144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3216269568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3222388992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3228508416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3230867712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3236987136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3239346432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3245465856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3251585280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3257704704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3260064000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3266183424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3272302848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3278422272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3280781568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3286900992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3289260288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3291619584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3293978880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3296338176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3302457600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3304816896, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3307176192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3309535488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3311894784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3318014208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3320373504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3322732800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 3, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3328852224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3334971648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3341091072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3347210496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3353329920, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3359449344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3361808640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3364167936, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3370287360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3376406784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3382526208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3388645632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3391004928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 3, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3397124352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3399483648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3405603072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3411722496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3414081792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3420201216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3422560512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3424919808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 3, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3431039232, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3437158656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3443278080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3445637376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 3, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3447996672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3450355968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3456475392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3458834688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 3, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3464954112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3467313408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3469672704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3472032000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3478151424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3480510720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3486630144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 3, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3492749568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3498868992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3504988416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3511107840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3517227264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3523346688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3529466112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3535585536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3541704960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3547824384, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3553943808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3556303104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3558662400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3561021696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3567141120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3573260544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3579379968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3581739264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3584098560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3590217984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3592577280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3594936576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3601056000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3603415296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3609534720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3611894016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3618013440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3624132864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3630252288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3632611584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 3, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3634970880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3637330176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3639689472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3642048768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3648168192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3654287616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3656646912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3659006208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3665125632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 3, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3667484928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3673604352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 3, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3675963648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3678322944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 3, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3680682240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3686801664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 3, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3689160960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3695280384, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3701399808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3703759104, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3709878528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3715997952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3722117376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3724476672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 3, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3726835968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3732955392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3735314688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3741434112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3747553536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3749912832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3752272128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3758391552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3764510976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3766870272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 3, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3769229568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3775348992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3777708288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3780067584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3782426880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3788546304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3794665728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3797025024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3799384320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3801743616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3804102912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3806462208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3812581632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3814940928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3817300224, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3819659520, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3822018816, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3824378112, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3830497536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3832856832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3835216128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 3, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3841335552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 3, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3843694848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3846054144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 3, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3852173568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3858292992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 3, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3860652288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3863011584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3865370880, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3867730176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3870089472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3872448768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3878568192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3880927488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3887046912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3889406208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3895525632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 3, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3901645056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3904004352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3910123776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3912483072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3914842368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 3, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3917201664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 3, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3919560960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3921920256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3924279552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3926638848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3928998144, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3931357440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3933716736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3939836160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3942195456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3944554752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3950674176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3953033472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3955392768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3957752064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3960111360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3962470656, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3964829952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3967189248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3969548544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3971907840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3974267136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3976626432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 3, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3978985728, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3981345024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3983704320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3986063616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 3, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3988422912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3990782208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3996901632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 3, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 3999260928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4005380352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4007739648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 3, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4013859072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4016218368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 3, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4018577664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4020936960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4023296256, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4029415680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4031774976, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4037894400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4044013824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4046373120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4048732416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4051091712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4053451008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4059570432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4061929728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4064289024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 3, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4070408448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 3 } }, { "memory": 4076527872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 3, "layer.31.mlp.down_proj": 4 } }, { "memory": 4082647296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4088766720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4091126016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4097245440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 4, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4103364864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4109484288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4115603712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4117963008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4120322304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4122681600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4128801024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4134920448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4141039872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4147159296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4153278720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4159398144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4161757440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4167876864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4170236160, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4176355584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4178714880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4181074176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 4, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4187193600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4189552896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4195672320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4198031616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4204151040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4210270464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4212629760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4214989056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4217348352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4223467776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4229587200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4231946496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4238065920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4240425216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4242784512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4245143808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4251263232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4253622528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4259741952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4265861376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4268220672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4274340096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4276699392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4282818816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4285178112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4287537408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4293656832, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4296016128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 4, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4298375424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 4, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4300734720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 4, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4306854144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4312973568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4315332864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4321452288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4323811584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4329931008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4332290304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4334649600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4340769024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 4, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4343128320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4345487616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4351607040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4353966336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4356325632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4362445056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4368564480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4370923776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4377043200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4379402496, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4381761792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4387881216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 4, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4390240512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4396359936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4402479360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4408598784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4410958080, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4417077504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 4, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4419436800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 4, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4425556224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4427915520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4430274816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4436394240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4442513664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4444872960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 4, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4447232256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4449591552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 4, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4455710976, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4458070272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4460429568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4466548992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4472668416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 4, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4475027712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4477387008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4483506432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4485865728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4488225024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4490584320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4496703744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4502823168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4505182464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4507541760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4513661184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4516020480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4522139904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4524499200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4530618624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4536738048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4542857472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 4, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4548976896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4555096320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4557455616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4559814912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4562174208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4564533504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4570652928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4573012224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4575371520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4581490944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4583850240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4589969664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4592328960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 4, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4594688256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 4, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4597047552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4599406848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4601766144, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4604125440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4606484736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4608844032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4611203328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4613562624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4619682048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4622041344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4628160768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4634280192, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4636639488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4642758912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4648878336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 4, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4651237632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4657357056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4663476480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4669595904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4671955200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4678074624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4680433920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4682793216, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4685152512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4687511808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4689871104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 4 } }, { "memory": 4695990528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 4, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4698349824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4704469248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 4, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4706828544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4709187840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4711547136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4713906432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 4, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4720025856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4726145280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4732264704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4738384128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4740743424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4743102720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4745462016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4751581440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4753940736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 4, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4756300032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 4, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4762419456, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4768538880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4770898176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4777017600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4783137024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4789256448, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4791615744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4797735168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4800094464, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4806213888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 4, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4808573184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4810932480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4813291776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4815651072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4818010368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4824129792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4826489088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4828848384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4834967808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4841087232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4843446528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4845805824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 4, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4851925248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4854284544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4860403968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4862763264, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4865122560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4867481856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4869841152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4872200448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4874559744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4876919040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4883038464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4889157888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4891517184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4897636608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4903756032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 4, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4906115328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4908474624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4914594048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 4, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4916953344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4919312640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4921671936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4924031232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4930150656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4932509952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4934869248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4937228544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4939587840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4941947136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4948066560, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4950425856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4952785152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4955144448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4957503744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4959863040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 4, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4965982464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 4, "layer.31.mlp.down_proj": 5 } }, { "memory": 4972101888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4974461184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4976820480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4982939904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4989059328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4991418624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4993777920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 4999897344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5002256640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5004615936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5006975232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5009334528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5015453952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5021573376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5023932672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5030052096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5032411392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5034770688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 5, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5037129984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5043249408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5049368832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 5, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5051728128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5054087424, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5056446720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5058806016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5061165312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5067284736, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5073404160, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5075763456, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5081882880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5088002304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5094121728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5096481024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 5, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5102600448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5104959744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5111079168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5113438464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5119557888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5125677312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5128036608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 5, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5134156032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5136515328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5138874624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5144994048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5147353344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 5, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5149712640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5155832064, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5158191360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5164310784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5166670080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5172789504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5178908928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 5, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5181268224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5187387648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5193507072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5195866368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5198225664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5200584960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 5, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5206704384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5212823808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5218943232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5225062656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 5, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5231182080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5237301504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5239660800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 5, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5242020096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5248139520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5254258944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 5, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5260378368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5266497792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5268857088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5274976512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5281095936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5283455232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5285814528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5291933952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5294293248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5300412672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5306532096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5312651520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5315010816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5321130240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 5, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5323489536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5325848832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5331968256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5334327552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5340446976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 5, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5342806272, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5348925696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5351284992, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5353644288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5356003584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5362123008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5364482304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5366841600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5369200896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 5, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5371560192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 5, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5373919488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5376278784, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5378638080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 5, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5384757504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 5, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5387116800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5393236224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5399355648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5405475072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5407834368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5410193664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5412552960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5418672384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 5, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5424791808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5430911232, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5437030656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5443150080, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5449269504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5451628800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5457748224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5460107520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 5, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5462466816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 5, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5464826112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5470945536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5473304832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5479424256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5485543680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 5, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5487902976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5494022400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5496381696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5498740992, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5504860416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5507219712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5509579008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5511938304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5514297600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5516656896, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5522776320, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5528895744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5531255040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5533614336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5539733760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5545853184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5551972608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 5, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5558092032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5560451328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5562810624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 5, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5565169920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5567529216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5569888512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 5, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5572247808, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5574607104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5580726528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5583085824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5585445120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5591564544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5593923840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5596283136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5598642432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5601001728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5607121152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5609480448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5611839744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5614199040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5616558336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5618917632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5625037056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5627396352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5633515776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5635875072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5638234368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5640593664, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5642952960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5645312256, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5651431680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5657551104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5659910400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5666029824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5668389120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5674508544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5676867840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5679227136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5681586432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5683945728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5690065152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5692424448, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5694783744, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5697143040, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 5, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5699502336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5701861632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5707981056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5714100480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5716459776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5718819072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5724938496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5727297792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5733417216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5735776512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5738135808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5740495104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5746614528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5748973824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5751333120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 5, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5757452544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5759811840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5762171136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5764530432, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5766889728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5769249024, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5775368448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 5, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5781487872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5787607296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5789966592, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5792325888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5794685184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5797044480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5803163904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5809283328, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5811642624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5814001920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5820121344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5822480640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5828600064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5830959360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5837078784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5843198208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 5, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5849317632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 5, "layer.31.mlp.down_proj": 5 } }, { "memory": 5855437056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 5 } }, { "memory": 5861556480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5863915776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5870035200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5876154624, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5878513920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5880873216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5883232512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 6, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5889351936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 6, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5895471360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5901590784, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5907710208, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5913829632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5919949056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5926068480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5932187904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 6, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5934547200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5936906496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5943025920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 6, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5945385216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5947744512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 6, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5950103808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5956223232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 6, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5958582528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5964701952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5967061248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5969420544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5971779840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5974139136, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 6, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5980258560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5982617856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5988737280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5991096576, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5993455872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 5999575296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6001934592, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6008054016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6014173440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6020292864, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6022652160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6028771584, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6031130880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6033490176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6039609600, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6041968896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6044328192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6046687488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6052806912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 6, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6058926336, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6061285632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 6, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6063644928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6069764352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6075883776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6078243072, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6084362496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6086721792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6092841216, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6098960640, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6105080064, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6107439360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6109798656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6115918080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6122037504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6124396800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6126756096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6129115392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6131474688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6137594112, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6143713536, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6149832960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6155952384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6162071808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6168191232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6174310656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6180430080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6182789376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 6, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6185148672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6187507968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6189867264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 6, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6192226560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6194585856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6200705280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6206824704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6209184000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6215303424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 6, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6217662720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6220022016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6226141440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6228500736, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6234620160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6240739584, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6246859008, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6249218304, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6255337728, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6257697024, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6263816448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6266175744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6268535040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6274654464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6277013760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6279373056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6285492480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6287851776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6293971200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6296330496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6298689792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6301049088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6303408384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6309527808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6311887104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6318006528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6320365824, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6322725120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6328844544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 6, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6334963968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6337323264, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6343442688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6345801984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6348161280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6354280704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6356640000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6358999296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6361358592, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6367478016, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6369837312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6372196608, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6374555904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6376915200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6379274496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6381633792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 6, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6383993088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6386352384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6392471808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6398591232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6404710656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6407069952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6409429248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6411788544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6414147840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6416507136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 6, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6418866432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6424985856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6431105280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6433464576, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 6, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6435823872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6438183168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6440542464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 6, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6442901760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 6, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6445261056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6451380480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6453739776, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6456099072, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6458458368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6460817664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6463176960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6465536256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6471655680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6474014976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6480134400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6482493696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6488613120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6490972416, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 6, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6493331712, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6495691008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6498050304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6500409600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6502768896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6505128192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6507487488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6513606912, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6519726336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6525845760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 6, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6531965184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 6, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6538084608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6540443904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6542803200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6548922624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 6, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6555042048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6557401344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6559760640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6562119936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6568239360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6570598656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6572957952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6579077376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6585196800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6587556096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6589915392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6596034816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6598394112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6600753408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6603112704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6609232128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6611591424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6617710848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6620070144, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6622429440, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6628548864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 6, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6630908160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6633267456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6635626752, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6641746176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6647865600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 6, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6650224896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6652584192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6658703616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6661062912, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6663422208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6665781504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6671900928, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6678020352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6684139776, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6690259200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6692618496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6694977792, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6697337088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6703456512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 6, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6709575936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6711935232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6714294528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6720413952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6726533376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 6, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6732652800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6738772224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 6 } }, { "memory": 6744891648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 6, "layer.31.mlp.down_proj": 7 } }, { "memory": 6751011072, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6753370368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6755729664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6758088960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6764208384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6766567680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6768926976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6771286272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6777405696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6783525120, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6789644544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6792003840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 7, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6794363136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6800482560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6802841856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 7, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6805201152, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 7, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6807560448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6809919744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6812279040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6818398464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6820757760, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6823117056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6829236480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6835355904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6837715200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6840074496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6846193920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 7, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6848553216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6854672640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6857031936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6863151360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 7, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6865510656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6871630080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 7, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6877749504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6880108800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6886228224, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6888587520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6890946816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6897066240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6899425536, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6905544960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6911664384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6914023680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6920143104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 7, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6926262528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6932381952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6938501376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6940860672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 7, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6946980096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6953099520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6959218944, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6965338368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6971457792, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6973817088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6979936512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6982295808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6984655104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 7, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6987014400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 7, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6989373696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 7, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6991732992, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6994092288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6996451584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 6998810880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7001170176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7007289600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 7, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7013409024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7015768320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7018127616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7020486912, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7026606336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7028965632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7035085056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7037444352, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7039803648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7042162944, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7044522240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7050641664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7056761088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7062880512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7068999936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 7, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7071359232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 7, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7077478656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7079837952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7082197248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7084556544, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7090675968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7093035264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7095394560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7097753856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7103873280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7109992704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7116112128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 7, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7118471424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7120830720, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7126950144, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7133069568, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 7, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7135428864, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7137788160, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7143907584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7150027008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7156146432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7162265856, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 7, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7164625152, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7166984448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7169343744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7175463168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7177822464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7183941888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7186301184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7192420608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7198540032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7200899328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7203258624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7205617920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7211737344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7214096640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7220216064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7222575360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7228694784, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7231054080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7233413376, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7235772672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 7, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7238131968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7240491264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7246610688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7252730112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7255089408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7257448704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7263568128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7265927424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7268286720, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7270646016, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7273005312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7275364608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 7, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7277723904, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7280083200, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7282442496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7288561920, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7290921216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7297040640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7299399936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7301759232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7304118528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7310237952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7312597248, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7314956544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7317315840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7323435264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7329554688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7331913984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7334273280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7340392704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7342752000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7345111296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7347470592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7349829888, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7355949312, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7358308608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7360667904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7366787328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7369146624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7371505920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7373865216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7379984640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7382343936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7384703232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7387062528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7389421824, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7391781120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7397900544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7404019968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7410139392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7416258816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7418618112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7420977408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 7, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7427096832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7433216256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7435575552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7441694976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 7, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7444054272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7450173696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7456293120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7458652416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7464771840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7470891264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7473250560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7479369984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7481729280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7484088576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7490208000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 7, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7496327424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7502446848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7504806144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7507165440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7513284864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 7, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7515644160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7518003456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 7, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7524122880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7530242304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7532601600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7534960896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7541080320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 7, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7547199744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7549559040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7551918336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7558037760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7564157184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7570276608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7572635904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7574995200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7577354496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7579713792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7585833216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7588192512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7590551808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7596671232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7602790656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7605149952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7607509248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 7, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7613628672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7619748096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7622107392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 7, "layer.31.mlp.down_proj": 7 } }, { "memory": 7628226816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 7, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 7 } }, { "memory": 7634346240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 7 } }, { "memory": 7640465664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7642824960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7645184256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7651303680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7653662976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7659782400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7662141696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7664500992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7670620416, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7676739840, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 8, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7682859264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 8, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7685218560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7691337984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 8, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7697457408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7699816704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7702176000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7708295424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 8, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7710654720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7716774144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7719133440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 8, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7721492736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7727612160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7729971456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 8, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7732330752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 8, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7734690048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7737049344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7743168768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7745528064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7751647488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7757766912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7763886336, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7770005760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7772365056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7778484480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7784603904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 8, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7786963200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7793082624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7795441920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7797801216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7803920640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7806279936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 8, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7812399360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7814758656, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7820878080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 8, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7826997504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7829356800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7835476224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7837835520, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7843954944, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7850074368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7856193792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7858553088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7860912384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7867031808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7869391104, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 8, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7875510528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7881629952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7883989248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7886348544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7892467968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7894827264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7900946688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7903305984, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7909425408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 8, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7911784704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7917904128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7920263424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 8, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7926382848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7928742144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7931101440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7933460736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7939580160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7941939456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7944298752, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 8, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7946658048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7949017344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7955136768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7957496064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 8, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7963615488, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 8, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7965974784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7968334080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7974453504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7980572928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7982932224, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7989051648, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7995171072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7997530368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 7999889664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8006009088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8008368384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8010727680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8013086976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8019206400, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8021565696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8027685120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8033804544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8039923968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8042283264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8044642560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8047001856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8049361152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8055480576, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8057839872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8060199168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 8, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8062558464, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8064917760, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8067277056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8073396480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8075755776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8078115072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8084234496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8086593792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 8, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8088953088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8095072512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 8, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8097431808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8099791104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8105910528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8108269824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8110629120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8112988416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8115347712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8117707008, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8123826432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8126185728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8128545024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8130904320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8137023744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8139383040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8141742336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8144101632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8146460928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 8, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8152580352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8158699776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8161059072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8163418368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8165777664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 8, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8171897088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8178016512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8180375808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8182735104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8188854528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8191213824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8193573120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8199692544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8202051840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8208171264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8214290688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8220410112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8222769408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8228888832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8231248128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8233607424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8235966720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 8, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8238326016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8240685312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8243044608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8245403904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 8, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8247763200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8250122496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8256241920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8262361344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8264720640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8267079936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8269439232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8275558656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8281678080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8284037376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8286396672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8288755968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8291115264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8293474560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 8, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8295833856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8298193152, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8300552448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8302911744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8305271040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8311390464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8313749760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8319869184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8325988608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8328347904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8330707200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8336826624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8339185920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8341545216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8343904512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8346263808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8348623104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8350982400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8357101824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8363221248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8369340672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8375460096, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8381579520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8383938816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8386298112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8388657408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8394776832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8400896256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8407015680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8409374976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8415494400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8417853696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 8, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8420212992, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8426332416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8432451840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8438571264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8440930560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8447049984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8449409280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8455528704, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8461648128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8467767552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8470126848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8472486144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 8, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8478605568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8484724992, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8487084288, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8493203712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8499323136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8505442560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 8 } }, { "memory": 8511561984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 8517681408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 8523800832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 8, "layer.31.mlp.down_proj": 9 } }, { "memory": 8529920256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 9, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8536039680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8538398976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8540758272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8546877696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8549236992, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8555356416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8557715712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8563835136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8566194432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8572313856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8578433280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8580792576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8583151872, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8589271296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 9, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8595390720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8597750016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8603869440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8606228736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8608588032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8610947328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8613306624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8619426048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8621785344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8627904768, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8630264064, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 9, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8632623360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8634982656, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 9, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8641102080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 9, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8647221504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8649580800, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8651940096, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8658059520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 9, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8660418816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8666538240, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8672657664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8678777088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8684896512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8691015936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8697135360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8703254784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 9, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8705614080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 9, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8707973376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8710332672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 9, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8712691968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8718811392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8721170688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8727290112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8729649408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8732008704, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8734368000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8736727296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8739086592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8745206016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8747565312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8753684736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 9, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8756044032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 9, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8758403328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8764522752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8770642176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8773001472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8779120896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8785240320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8791359744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8793719040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8796078336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8798437632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8800796928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8806916352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8809275648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 9, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8811634944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8817754368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8820113664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8822472960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8824832256, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 9, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8827191552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8829550848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8835670272, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8841789696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8844148992, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 9, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8846508288, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 9, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8852627712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8858747136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8861106432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8863465728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 9, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8865825024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8871944448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 9, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8874303744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8876663040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8882782464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8888901888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8891261184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8893620480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8899739904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8905859328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8908218624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 9, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8910577920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8912937216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8915296512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8917655808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8920015104, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8926134528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8928493824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 9, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8930853120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8936972544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8939331840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8945451264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8947810560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8953929984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8956289280, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8958648576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8964768000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8967127296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8973246720, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8975606016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8981725440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8987844864, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8990204160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8996323584, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 8998682880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9004802304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9007161600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9013281024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9019400448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 9, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9025519872, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9031639296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9033998592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9036357888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9038717184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9044836608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9047195904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9049555200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9051914496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9054273792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9060393216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9066512640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9068871936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9071231232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9073590528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9075949824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9082069248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9084428544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9090547968, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9092907264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9095266560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9097625856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9099985152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9106104576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9108463872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9114583296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9116942592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9123062016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9125421312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9131540736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9137660160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9143779584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9146138880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9152258304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9158377728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9160737024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9163096320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9165455616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9167814912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9170174208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9176293632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9178652928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9181012224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9187131648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 9, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9189490944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9191850240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9194209536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9200328960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9206448384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9208807680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9211166976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 9, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9217286400, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9219645696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9225765120, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9228124416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9230483712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9232843008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9235202304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9237561600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9239920896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9242280192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9244639488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 9, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9250758912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9253118208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9259237632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9261596928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9263956224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9266315520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9268674816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9271034112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9277153536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9283272960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9289392384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9291751680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9294110976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9296470272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9302589696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9304948992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9307308288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 9, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9313427712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9319547136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9325666560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 9, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9328025856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9330385152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9336504576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9342624000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 9, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9348743424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9351102720, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9357222144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9359581440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9365700864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9368060160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9374179584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9380299008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9386418432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9392537856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9394897152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9401016576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 9, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9407136000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 9, "layer.31.mlp.down_proj": 9 } }, { "memory": 9413255424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 9 } }, { "memory": 9419374848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9421734144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9424093440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9426452736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9428812032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 10, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9434931456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9437290752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9439650048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9445769472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9448128768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9454248192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9456607488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9462726912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 10, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9465086208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9467445504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9473564928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9475924224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9478283520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9480642816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9483002112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9489121536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9495240960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9501360384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9507479808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9513599232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9519718656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9525838080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9528197376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9530556672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9536676096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 10, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9539035392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9541394688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 10, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9543753984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9546113280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 10, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9552232704, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9558352128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 10, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9560711424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9566830848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9569190144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9571549440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9577668864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9583788288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9589907712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 10, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9592267008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9594626304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9600745728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9603105024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9605464320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9607823616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9613943040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9616302336, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9622421760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 10, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9624781056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9630900480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9637019904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 10, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9643139328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 10, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9645498624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9647857920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9653977344, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9656336640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9658695936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9664815360, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9670934784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9673294080, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9675653376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9678012672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9684132096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9686491392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9688850688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 10, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9691209984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 10, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9693569280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9695928576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9702048000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9708167424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9714286848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9716646144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 10, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9719005440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9725124864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9727484160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9733603584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 10, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9735962880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9738322176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9740681472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9746800896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9749160192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9751519488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9757638912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9759998208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 10, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9762357504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9768476928, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9770836224, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9773195520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9775554816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9777914112, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9780273408, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 10, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9786392832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9788752128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9794871552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 10, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9797230848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9799590144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9801949440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9804308736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9810428160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9812787456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9818906880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9821266176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9823625472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9829744896, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9832104192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9838223616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9844343040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9850462464, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9852821760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9855181056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9857540352, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9859899648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9862258944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9864618240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9866977536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9869336832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9875456256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9881575680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9883934976, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9886294272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9892413696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 10, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9894772992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9897132288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9899491584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 10, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9905611008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9907970304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9910329600, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9916449024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9918808320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9921167616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9923526912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9925886208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9928245504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 10, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9930604800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9932964096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 10, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9939083520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9945202944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9947562240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9953681664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9959801088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9962160384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9964519680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9970639104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9972998400, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9975357696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9977716992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9983836416, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9989955840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9996075264, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 9998434560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10000793856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10003153152, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10009272576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10015392000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10021511424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10027630848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10033750272, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10036109568, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10038468864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10040828160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10046947584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10049306880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10051666176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10054025472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10060144896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10062504192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10068623616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 10, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10070982912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10077102336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10083221760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10085581056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10087940352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10090299648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10092658944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10098778368, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10101137664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10103496960, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10109616384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10115735808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10118095104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10120454400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10126573824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10132693248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10138812672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10141171968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10147291392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 10, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10153410816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10155770112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10158129408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 10, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10164248832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10166608128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10172727552, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10175086848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10177446144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10183565568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10185924864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10188284160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10190643456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10193002752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10195362048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 10, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10201481472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10203840768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10206200064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10212319488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10214678784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 10, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10220798208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10223157504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10229276928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10235396352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10241515776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10247635200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10253754624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10259874048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10265993472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 10, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10272112896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10278232320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10284351744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 10, "layer.31.mlp.down_proj": 10 } }, { "memory": 10290471168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10 } }, { "memory": 10296590592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10 } }, { "memory": 10302710016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 10 } }, { "memory": 10308829440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10311188736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10313548032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 11, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10319667456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10322026752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10328146176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10334265600, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10336624896, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10338984192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10341343488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10347462912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10353582336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 11, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10359701760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10362061056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10368180480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10370539776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 11, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10376659200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10379018496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10385137920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10391257344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10393616640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10395975936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10402095360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10408214784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10410574080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10412933376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10415292672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10417651968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 11, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10420011264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10426130688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10428489984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10434609408, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10436968704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10439328000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10445447424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 11, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10451566848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10457686272, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 11, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10460045568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 11, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10462404864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 11, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10464764160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10467123456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10473242880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10479362304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10485481728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10487841024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10490200320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10492559616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10494918912, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10501038336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10503397632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10509517056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10515636480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10521755904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 11, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10524115200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 11, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10530234624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 11, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10532593920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10538713344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10544832768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10550952192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10553311488, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10555670784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10558030080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10564149504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10566508800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10568868096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10571227392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10573586688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10575945984, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 11, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10578305280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10584424704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10590544128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10592903424, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10599022848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10605142272, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 11, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10607501568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10609860864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10615980288, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10622099712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10624459008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10630578432, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10632937728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10635297024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10637656320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10643775744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 11, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10646135040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10652254464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10658373888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10664493312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10666852608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10672972032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10679091456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 11, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10681450752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10683810048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10686169344, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10692288768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10698408192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10704527616, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10710647040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10713006336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10719125760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10725245184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10731364608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 11, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10733723904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10736083200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 11, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10738442496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10740801792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10743161088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10749280512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 11, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10751639808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10757759232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10760118528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10766237952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10768597248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10774716672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10777075968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10779435264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10781794560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 11, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10784153856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10786513152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10788872448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10794991872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10797351168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10799710464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10805829888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10811949312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10814308608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10816667904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10819027200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10821386496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10823745792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10826105088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10828464384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10830823680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10836943104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10843062528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10845421824, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10847781120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10850140416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 11, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10856259840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10862379264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10864738560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10867097856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10873217280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10875576576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10881696000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10884055296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10890174720, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10892534016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10898653440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10901012736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10903372032, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10905731328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10908090624, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10914210048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10916569344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10922688768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10925048064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10927407360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10933526784, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10939646208, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10945765632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10951885056, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10958004480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10960363776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10966483200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10968842496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10974961920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10981081344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10983440640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10985799936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10988159232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 11, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10990518528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10992877824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 11, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 10998997248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11001356544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11003715840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11006075136, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11012194560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11014553856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 11, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11020673280, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11026792704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11029152000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11031511296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11037630720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11043750144, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11046109440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11048468736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11054588160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11056947456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11059306752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11061666048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11067785472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11070144768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11072504064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11074863360, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11077222656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 11, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11079581952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11081941248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11084300544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11086659840, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11089019136, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11095138560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11101257984, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11107377408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11109736704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11112096000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11114455296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11116814592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11122934016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11125293312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11131412736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 11, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11137532160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11139891456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11142250752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11148370176, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11150729472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 11, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11156848896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11159208192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11165327616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11171447040, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11177566464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11179925760, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11186045184, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 11, "layer.31.mlp.down_proj": 11 } }, { "memory": 11192164608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 11 } }, { "memory": 11198284032, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11200643328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11203002624, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11205361920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 12, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11211481344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11213840640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11216199936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11218559232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 12, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11224678656, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11230798080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11233157376, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11235516672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 12, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11237875968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11240235264, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11246354688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11252474112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11258593536, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11260952832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11267072256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11273191680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11275550976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11277910272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11280269568, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11282628864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11284988160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11287347456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11289706752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11292066048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11294425344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11296784640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11299143936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11301503232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11303862528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11309981952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11312341248, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11318460672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11320819968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11326939392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 12, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11329298688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11335418112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 12, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11337777408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 12, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11340136704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11346256128, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11352375552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11358494976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11364614400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 12, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11366973696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11369332992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 12, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11371692288, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11377811712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11383931136, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11386290432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 12, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11392409856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11398529280, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 12, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11400888576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11407008000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11409367296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11411726592, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11417846016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11420205312, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11422564608, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11424923904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 12, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11427283200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 12, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11429642496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11432001792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11434361088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11436720384, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 12, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11439079680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11445199104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11447558400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11449917696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11456037120, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11458396416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11460755712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11463115008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11469234432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11471593728, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11477713152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11483832576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 12, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11486191872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11492311296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11498430720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11504550144, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11506909440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11513028864, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11515388160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 12, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11521507584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11523866880, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11526226176, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11528585472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11534704896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11537064192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11543183616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11545542912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11547902208, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11550261504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11552620800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11554980096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11557339392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11559698688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11565818112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11571937536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11574296832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 12, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11576656128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11582775552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11588894976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11591254272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11597373696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11599732992, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11602092288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 12, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11608211712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11614331136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11620450560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 12, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11622809856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11625169152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11631288576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11637408000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11643527424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11645886720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11648246016, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11650605312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11652964608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11655323904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11657683200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11660042496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11662401792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 12, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11664761088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11667120384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11673239808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11675599104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11677958400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11684077824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11686437120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11692556544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11698675968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11701035264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11703394560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11709513984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11711873280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11714232576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11716591872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11718951168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11721310464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11727429888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11733549312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 12, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11739668736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11742028032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11744387328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11750506752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11752866048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11758985472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11761344768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11767464192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11769823488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11775942912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11782062336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11784421632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 12, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11786780928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11789140224, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11795259648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11797618944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11799978240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11802337536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11808456960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11810816256, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11813175552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11815534848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11821654272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11824013568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11826372864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11828732160, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11834851584, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11837210880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11843330304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11849449728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11851809024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11857928448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11860287744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11866407168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11868766464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11871125760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11873485056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11875844352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11878203648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11880562944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11886682368, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 12, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11892801792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11898921216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11905040640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 12, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11911160064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 12, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11917279488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11919638784, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11921998080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 12, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11924357376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11926716672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11929075968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11935195392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11937554688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11939913984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11946033408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11948392704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11954512128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11956871424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11962990848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11969110272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11975229696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11981349120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11987468544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11993587968, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 11999707392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12002066688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12008186112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12014305536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12020424960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12026544384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12032663808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12038783232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12044902656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12051022080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12057141504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 12 } }, { "memory": 12063260928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 12069380352, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 12, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 12075499776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 12081619200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 12, "layer.31.mlp.down_proj": 13 } }, { "memory": 12087738624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12090097920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12092457216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12094816512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12097175808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12103295232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12109414656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12111773952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12117893376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12124012800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 13, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12126372096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12128731392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12134850816, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12140970240, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12143329536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 13, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12145688832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12148048128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 13, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12150407424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12156526848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12162646272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12165005568, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12167364864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12173484288, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12179603712, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12181963008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12188082432, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12190441728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12192801024, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12198920448, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 13, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12201279744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12207399168, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12213518592, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12215877888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12221997312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12224356608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12230476032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12232835328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12235194624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12241314048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 13, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12243673344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 13, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12249792768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12252152064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12258271488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12264390912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12270510336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12276629760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12282749184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12288868608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 13, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12294988032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12301107456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12307226880, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12309586176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12311945472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 13, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12318064896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12320424192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12326543616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12332663040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12335022336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12341141760, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12343501056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12349620480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12351979776, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12358099200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12364218624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12370338048, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12372697344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 13, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12375056640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12381176064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12383535360, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12389654784, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12392014080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12394373376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12396732672, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12402852096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12408971520, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12415090944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12421210368, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12427329792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12433449216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12439568640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12445688064, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12448047360, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12454166784, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12456526080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12458885376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 13, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12465004800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12467364096, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12469723392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12472082688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12474441984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12480561408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12486680832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 13, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12489040128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 13, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12491399424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12493758720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12499878144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 13, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12502237440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 13, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12504596736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12506956032, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12509315328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12511674624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12514033920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12516393216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12518752512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12521111808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12523471104, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12529590528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12531949824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12538069248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12540428544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12542787840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12548907264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12551266560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12557385984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12559745280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12562104576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 13, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12564463872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12570583296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12576702720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12579062016, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12581421312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12583780608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12586139904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 13, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12588499200, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 13, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12590858496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12593217792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12595577088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 13, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12597936384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12600295680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12602654976, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12608774400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12614893824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12617253120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12623372544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12625731840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12631851264, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 13, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12637970688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 13, "layer.31.mlp.down_proj": 13 } }, { "memory": 12644090112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12646449408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 13, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12652568832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12654928128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12661047552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12663406848, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 13, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12665766144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12671885568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12678004992, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12680364288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12682723584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12688843008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12691202304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 13, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12693561600, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12695920896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12702040320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12704399616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12710519040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12712878336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12715237632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12721357056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12723716352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12726075648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12728434944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12730794240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12733153536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12735512832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12737872128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12740231424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12742590720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12744950016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12751069440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12753428736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12759548160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12765667584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12768026880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12770386176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12772745472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12778864896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12781224192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12783583488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12789702912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12792062208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12794421504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12796780800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12802900224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12805259520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12807618816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12809978112, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12812337408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12814696704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 13, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12820816128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12823175424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12825534720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12831654144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12837773568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12840132864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12842492160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12848611584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12854731008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12857090304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 13, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12863209728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12865569024, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12871688448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12874047744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12880167168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12882526464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12888645888, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12894765312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12897124608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12903244032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12909363456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12911722752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12914082048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12920201472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12926320896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12932440320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12934799616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12937158912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12939518208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12941877504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12947996928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12950356224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 13, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12956475648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12958834944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12964954368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 13, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12971073792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 13 } }, { "memory": 12977193216, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12979552512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12981911808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12984271104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12990390528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12992749824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 12998869248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13001228544, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 14, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13007347968, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13009707264, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13012066560, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13014425856, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13016785152, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13022904576, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13025263872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13031383296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13037502720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13039862016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13045981440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13048340736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13050700032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13053059328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13055418624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13061538048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13063897344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13066256640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 14, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13068615936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13070975232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13077094656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13083214080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13089333504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13095452928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13101572352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 14, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13103931648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 14, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13110051072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13112410368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13114769664, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 14, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13120889088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13127008512, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13133127936, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13135487232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13141606656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13143965952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13150085376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13156204800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13158564096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 14, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13160923392, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13167042816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13173162240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13175521536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13177880832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13180240128, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13182599424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13184958720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 14, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13191078144, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13197197568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13199556864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 14, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13201916160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13208035584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 14, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13210394880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13212754176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 14, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13215113472, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 14, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13217472768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13223592192, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 14, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13225951488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13228310784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13234430208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 14, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13240549632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 14, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13242908928, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13245268224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13251387648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13257507072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13259866368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13265985792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13268345088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13270704384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13276823808, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13282943232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13285302528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13291421952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13297541376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 14, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13303660800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13309780224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13315899648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 14, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13322019072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 14, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13324378368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13330497792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13332857088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13335216384, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13337575680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13339934976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13346054400, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13348413696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13354533120, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13360652544, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13363011840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13365371136, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13367730432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13370089728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13372449024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13374808320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13377167616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13379526912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13385646336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13388005632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13390364928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13392724224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13398843648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13401202944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13403562240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13405921536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13412040960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13414400256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13416759552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13419118848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13421478144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13427597568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13429956864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13432316160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13438435584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13440794880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13443154176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13449273600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13451632896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13457752320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 14, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13460111616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13466231040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13468590336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13470949632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13477069056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13479428352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13485547776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13487907072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13490266368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13496385792, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13498745088, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13504864512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13507223808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13509583104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13511942400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13514301696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13516660992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13519020288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13525139712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13527499008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13529858304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 14, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13532217600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13534576896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13536936192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13539295488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13545414912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13551534336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13553893632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13560013056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 14, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13566132480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13572251904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13574611200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13580730624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13586850048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13589209344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13591568640, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13593927936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 14, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13600047360, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13606166784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13612286208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13618405632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13620764928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13626884352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13629243648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13631602944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13633962240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13636321536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13638680832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13644800256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13650919680, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13653278976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13659398400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 14, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13661757696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13667877120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13670236416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13676355840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13678715136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13681074432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13687193856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13689553152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13691912448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13694271744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13700391168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13702750464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13708869888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 14, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13711229184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13713588480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13715947776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13722067200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13728186624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13730545920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13732905216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13735264512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13741383936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 14, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13747503360, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13753622784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13755982080, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13762101504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13764460800, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 14 } }, { "memory": 13770580224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13776699648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13779058944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13781418240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 14, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13783777536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 14, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13786136832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13788496128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13794615552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13800734976, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13803094272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13805453568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 14, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13811572992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13817692416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13823811840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13829931264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13836050688, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13842170112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13848289536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13854408960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13860528384, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 14, "layer.31.mlp.down_proj": 15 } }, { "memory": 13866647808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13869007104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13871366400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13873725696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13879845120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13885964544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13888323840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 15, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13890683136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13893042432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13899161856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 15, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13905281280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13907640576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13913760000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13919879424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 15, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13922238720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13928358144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13930717440, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 15, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13933076736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 15, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13935436032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13937795328, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13940154624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13946274048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13952393472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13958512896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13964632320, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13966991616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13969350912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 15, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13971710208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13974069504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13976428800, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13982548224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13984907520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13991026944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13993386240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 13999505664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 15, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14001864960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14004224256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14006583552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14008942848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 15, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14015062272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14017421568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14019780864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14022140160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14028259584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 15, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14030618880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14032978176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14035337472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14041456896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14043816192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14049935616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14056055040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14058414336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14060773632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14066893056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14069252352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 15, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14071611648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14073970944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14080090368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14086209792, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14092329216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14098448640, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14104568064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 15, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14106927360, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14113046784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14119166208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14125285632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14127644928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14133764352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14136123648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14138482944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14144602368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14146961664, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14149320960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14155440384, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14161559808, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14163919104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14166278400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14168637696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14170996992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 15, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14177116416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14179475712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14185595136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14191714560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14197833984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14200193280, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14202552576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14208672000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14211031296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14217150720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14219510016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14225629440, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14231748864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14234108160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14236467456, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14242586880, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 15, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14248706304, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14254825728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14260945152, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14263304448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14269423872, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14275543296, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14281662720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14287782144, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14290141440, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14296260864, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14298620160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 15, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14304739584, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14307098880, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14309458176, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14311817472, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14317936896, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14320296192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 15, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14322655488, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 15, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14325014784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14327374080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14329733376, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14335852800, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14338212096, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 15, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14340571392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14346690816, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14349050112, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14351409408, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14357528832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 15, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14363648256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 15, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14369767680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14372126976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14378246400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14384365824, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14386725120, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14389084416, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14391443712, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14397563136, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 15, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14399922432, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14402281728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14408401152, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14410760448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14413119744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14419239168, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14421598464, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14423957760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14426317056, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14432436480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14434795776, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14440915200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14443274496, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 15, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14445633792, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14447993088, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14450352384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14452711680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14458831104, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14461190400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14463549696, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14465908992, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14468268288, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14474387712, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14476747008, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14479106304, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14485225728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14487585024, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14489944320, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14496063744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14498423040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14500782336, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14506901760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14509261056, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14511620352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14513979648, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14516338944, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 15, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14522458368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14528577792, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14530937088, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14537056512, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14543175936, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14549295360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14555414784, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14561534208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14563893504, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14570012928, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14572372224, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14578491648, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14580850944, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 15, "layer.31.mlp.down_proj": 15 } }, { "memory": 14586970368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 15, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14589329664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14591688960, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14597808384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14600167680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14602526976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14604886272, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14607245568, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14613364992, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14619484416, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14625603840, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14627963136, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14630322432, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14632681728, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14635041024, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14641160448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14643519744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14645879040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14648238336, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14654357760, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14660477184, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14662836480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14668955904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14671315200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14677434624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14683554048, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14685913344, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14688272640, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 15, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14694392064, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14696751360, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14699110656, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14701469952, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 15, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14703829248, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14709948672, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14716068096, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14722187520, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14724546816, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14726906112, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14729265408, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14735384832, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14741504256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14743863552, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 15, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14749982976, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 15 } }, { "memory": 14756102400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16, "layer.28.self_attn.q_proj": 16, "layer.28.self_attn.k_proj": 16, "layer.28.self_attn.v_proj": 16, "layer.28.self_attn.o_proj": 16, "layer.28.mlp.gate_proj": 16, "layer.28.mlp.up_proj": 16, "layer.28.mlp.down_proj": 16, "layer.29.self_attn.q_proj": 16, "layer.29.self_attn.k_proj": 16, "layer.29.self_attn.v_proj": 16, "layer.29.self_attn.o_proj": 16, "layer.29.mlp.gate_proj": 16, "layer.29.mlp.up_proj": 16, "layer.29.mlp.down_proj": 16, "layer.30.self_attn.q_proj": 16, "layer.30.self_attn.k_proj": 16, "layer.30.self_attn.v_proj": 16, "layer.30.self_attn.o_proj": 16, "layer.30.mlp.gate_proj": 16, "layer.30.mlp.up_proj": 16, "layer.30.mlp.down_proj": 16, "layer.31.self_attn.q_proj": 16, "layer.31.self_attn.k_proj": 16, "layer.31.self_attn.v_proj": 16, "layer.31.self_attn.o_proj": 16, "layer.31.mlp.gate_proj": 16, "layer.31.mlp.up_proj": 16, "layer.31.mlp.down_proj": 16 } } ]